#encoding: utf-8
# -*- coding: utf-8 -*-
import codecs
import os,sys
from st_common import *
from langconv import *
import cPickle
 

def LoadResult(work_file):
    fwd=codecs.open(work_file,'r',encoding='gbk')
    #fwd=codecs.open('D:/data/CDT7/svn.conll06','r',encoding='UTF8')
    raw=fwd.read()
    fwd.close()
    lines=raw.split('\n')
    
    corpor=[]
    line_vec=[]
    for ln in lines:
        word=ln.split()
            
        words=[ ''.join(wd.split('\n')) for wd in word if len(wd)>0]
        if len(words)==0 and len(line_vec)>0:
            if len(line_vec)>=1:
                corpor.append(line_vec)
            else:
                line_vec=[]
                continue
                #print 'error line_vec:'+'\t'.join(line_vec[0])
                
            line_vec=[]
        else:
            line_vec.append(words)
    return corpor


os.chdir('E:\\desktop\\wu-request\\NLPCC 2014 Shared Tasks Guidelines\\crfNER')
BASE_DATA_PATH  = 'D:\\data\\Shared_Task_NLPCC14\\'
BASE_KB_PATH = 'E:\\desktop\\wu-request\\NLPCC 2014 Shared Tasks Guidelines\\Chinese Entity Linking  SAMPLE DATA NLPCC2014_EL_sample\\'




if False: # segment
    seg_rs = LoadResult('people_daily_test_seg_result.txt')
    
    all_count = 0
    crt_count = 0
    for ln in seg_rs:
        all_count = all_count+ len(ln)
        for wd in ln:
            if wd[1]==wd[-1]:
                crt_count = crt_count+1.0
    print 'seg_result',crt_count/all_count
    
    segmented_corpus = []
    for ln in seg_rs:
        line = []
        word = ''
        for wdc in ln:
            if wdc[-1]=='B':
                if word:
                    line.append(word)
                word = wdc[0]
            if wdc[-1]=='I':
                word = word+wdc[0]
            if wdc[-1]=='E':
                word = word+wdc[0]
        if word:
            line.append(word)
        segmented_corpus.append(line)
        
    fwd=codecs.open('people_daily_test_seg_resnormal.txt','w',encoding='gbk')
    for ln in segmented_corpus:
        fwd.write('\n'.join(ln))
        fwd.write('\n')
    fwd.close()

if False: #named entity
    ne_rs = LoadResult('people_daily_netest_result.txt')
    
    all_count_no = 0
    crt_count_no = 0
    all_count_o = 0
    crt_count_o = 0
    for ln in ne_rs:
        #all_count = all_count+ len(ln)
        for wd in ln:
            if wd[2]!='O':
                all_count_no=all_count_no+1
                if wd[2]==wd[-1]:
                    crt_count_no = crt_count_no+1.0
            else:
                all_count_o=all_count_o+1
                if wd[2]==wd[-1]:
                    crt_count_o = crt_count_o+1.0
    print 'NE_result',crt_count_no/all_count_no,crt_count_o/all_count_o
    
    named_entity_corpus = []
    for ln in ne_rs:
        line = []
        word = ''
        word_tag = ''
        for wdc in ln:
            tag = wdc[-1]
            if tag[-1]=='O':
                if word:
                    line.append(word+'_'+ word_tag)
                word=''
                word_tag = ''
                line.append(wdc[0]+'_'+wdc[1])
            if tag[-1]=='B':
                if word:
                    line.append(word+'_'+ word_tag)
                word = wdc[0]
                word_tag = tag[0:-2]
            if tag[-1]=='I':
                word = word+wdc[0]
            if tag[-1]=='E':
                word = word+wdc[0]
        if word:
            line.append(word+' '+ word_tag)
        named_entity_corpus.append(line)
    
    fwd=codecs.open('people_daily_netest_resnormal.txt','w',encoding='gbk')
    for ln in named_entity_corpus:
        fwd.write('\t'.join(ln))
        fwd.write('\n')
    fwd.close()
    
if True: #Pos
    pos_rs = LoadResult('people_daily_test_result.txt')
    all_count = 0
    crt_count = 0
    for ln in pos_rs:
        all_count = all_count+ len(ln)
        for wd in ln:
            if wd[1]==wd[-1]:
                crt_count = crt_count+1.0
    print 'pos_result',crt_count/all_count

    